# -*- coding: utf-8 -*-
'''
Created on Feb 6, 2013

@author: LONG HOANG GIANG
'''
from CrawlerLib2 import commonlib, html2text
from urlparse import urljoin
import datetime
import re
import workerpool
from Model.storymodelv3 import StoryModel

URL = 'http://vinapro.net/vina-truyen/truyen-nguoi-lon/page/{0}/'


class TNL():
    
    def getDetail(self, url):
        tree = commonlib.loadweb(url, nocache=False).build_tree()
        detailNode = tree.xpath("//div[@class='entry-content']")
        if len(detailNode) == 0:
            tree = commonlib.loadweb(url, nocache=True).build_tree()
            detailNode = tree.xpath("//div[@class='entry-content']")
        detailNode = detailNode[0]
        commonlib.Etree.clean_elements(detailNode, 'div', {'class': 'addthis_toolbox addthis_default_style '})
        commonlib.Etree.clean_elements(detailNode, 'div', {'class': 'ratings hreview-aggregate'})
        commonlib.Etree.clean_elements(detailNode, 'div', {'class': 'page-link'})
        commonlib.Etree.clean_elements(detailNode, 'div', {'class': 'wp-pagenavi'})
        commonlib.Etree.clean_elements(detailNode, 'div', {'class': 'entry-tags'})
        content = html2text.html2text(commonlib.Etree.tostring(detailNode), w100=True, textonly=True).encode('utf-8').strip()
        print content
        content = re.sub(r"\n", "<br />", content)
        return content;
    
    def getListStory(self):
        data = commonlib.json_load('/longhoanggiang/json/tnl')
        if data == None:
            data = []
            for page in range(1, 14):
                url = URL.format(page)
                tree = commonlib.loadweb(url).build_tree()
                for node in tree.xpath("//div[contains(@id, 'post-')]"):
                    title = commonlib.toUpper(commonlib.stringify(node.xpath(".//h2[@class='entry-title']/a")))
                    print title
                    count = 1
                    childNodes = node.xpath(".//div[@class='entry-excerpt']//div[@class='page-link']/a")
                    for dpage in childNodes:
                        link = dpage.get('href', '')
                        if link == '': continue
                        link = urljoin(url, link)
                        if len(childNodes) > 1:
                            data.append({'title': title + " ({0})".format(count), 'url': link})
                        else:
                            data.append({'title': title, 'url': link})
                        count += 1
            commonlib.json_dump(data, '/longhoanggiang/json/tnl')
        
#        for item in data:
#            print item['title'], item['url']
#            
#        def loadWeb(item):
#            url = item['url']
#            commonlib.loadweb(url)
#        
#        pool = workerpool.WorkerPool(size=1)
#        pool.map(loadWeb, data)
#        pool.shutdown()
#        pool.wait()
        model = StoryModel(database)
        model.open(True)
        for item in data:
            detail = self.getDetail(item['url'])
            detail = '''<strong>{0}</strong><br /><hr /><br />{1}'''.format(item['title'], detail)
            model.add_story(item['title'], detail, 0)
        model.close()
                
                
if __name__ == '__main__':
    database = "/longhoanggiang/database/tnl"
    tnl = TNL()
    tnl.getListStory()
#    tnl.getDetail('http://vinapro.net/v%e1%bb%a3-ch%e1%bb%93ng-ba-ph%c6%b0%c6%a1ng/2/')
    
    
    print '> Finished at {0}'.format(datetime.datetime.now())